{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "Fu3OT8EoaUOE"
   },
   "outputs": [],
   "source": [
    "# import packages \n",
    "\n",
    "import numpy as np\n",
    "import os\n",
    "import datetime\n",
    "import time\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import h5py\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "# folder path and name\n",
    "project_path = os.getcwd()\n",
    "data_folder = os.path.join(project_path,\"data\")\n",
    "pred_folder = os.path.join(data_folder,'npy_before_hdf5')\n",
    "pv_data_path = os.path.join(data_folder,'pv_data','pv_output_valid.pkl')\n",
    "\n",
    "image_name_format = '%Y%m%d%H%M%S'\n",
    "\n",
    "# Operating parameter\n",
    "stack_height = 15 # 15 minute\n",
    "forecast_horizon = 5 # 5 minutes ahead forecast\n",
    "sampling_interval_all = [2]\n",
    "output_img_shape = [64, 64, 3]\n",
    "\n",
    "start_date = datetime.datetime(2017,1,1) #NOTE: Inclusive of start date\n",
    "end_date = datetime.datetime(2018,1,1) #NOTE: Exclusive of end date (only end up with 2017 data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "fwRci7HpaUOx"
   },
   "outputs": [],
   "source": [
    "def find_time_within_nparray(time_array,time_point):\n",
    "    probable_idx = np.searchsorted(time_array,time_point)\n",
    "    \n",
    "    # If the time point is after all the time in pv_data\n",
    "    if probable_idx == len(time_array):\n",
    "        return None   \n",
    "    \n",
    "    # See if the time point is actually a match \n",
    "    if time_array[probable_idx]== time_point: \n",
    "        return probable_idx\n",
    "        \n",
    "    else:\n",
    "        return None\n",
    "\n",
    "def find_time_within_pdseries(time_array,time_point):\n",
    "    probable_idx = np.searchsorted(time_array,time_point)\n",
    "    \n",
    "    # If the time point is after all the time in pv_data\n",
    "    if probable_idx == len(time_array):\n",
    "        return None   \n",
    "    \n",
    "    # See if the time point is actually a match \n",
    "    if time_array[probable_idx] == time_point: \n",
    "        return probable_idx\n",
    "        \n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "RPtJ7Nw-aUPC"
   },
   "outputs": [],
   "source": [
    "def store_trainval_test(all_times,image_log,image_pred,pv_log,pv_pred):\n",
    "        \n",
    "        # 计算训练集和测试集的分割点\n",
    "        train_size = int(len(all_times) * 0.9)  # 前 90%\n",
    "        test_size = len(all_times) - train_size  # 后 10%\n",
    "\n",
    "        # 划分训练集和测试集\n",
    "        train_times, test_times = all_times[:train_size], all_times[-test_size:]\n",
    "        train_images_log, test_images_log = image_log[:train_size], image_log[-test_size:]\n",
    "        train_images_pred, test_images_pred = image_pred[:train_size], image_pred[-test_size:]\n",
    "        train_pv_log, test_pv_log = pv_log[:train_size], pv_log[-test_size:]\n",
    "        train_pv_pred, test_pv_pred = pv_pred[:train_size], pv_pred[-test_size:]\n",
    "\n",
    "        np.save(os.path.join(pred_folder,'All','image_log_trainval.npy'), train_images_log)\n",
    "        np.save(os.path.join(pred_folder,'All','image_pred_trainval.npy'), train_images_pred)\n",
    "        np.save(os.path.join(pred_folder,'All','pv_log_trainval.npy'), train_pv_log)\n",
    "        np.save(os.path.join(pred_folder,'All','pv_pred_trainval.npy'),train_pv_pred)\n",
    "        np.save(f'./data/video_data/times_5min_trainval.npy',train_times)\n",
    "        np.save(f'./data/data_forecast/All/times_trainval.npy',train_times)\n",
    "\n",
    "        np.save(os.path.join(pred_folder,'All','image_log_test.npy'), test_images_log)\n",
    "        np.save(os.path.join(pred_folder,'All','image_pred_test.npy'), test_images_pred)\n",
    "        np.save(os.path.join(pred_folder,'All','pv_log_test.npy'), test_pv_log)\n",
    "        np.save(os.path.join(pred_folder,'All','pv_pred_test.npy'),test_pv_pred)\n",
    "        np.save(f'./data/video_data/times_5min_test.npy',test_times)\n",
    "        np.save(f'./data/data_forecast/All/times_test.npy',test_times)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "QH5ewDbBaUPT"
   },
   "outputs": [],
   "source": [
    "# Load in  high frequency data\n",
    "# the image here are ones that have corresponding PV value\n",
    "all_times = np.load(os.path.join(data_folder,'all_times_highfreq.npy'), allow_pickle=True)\n",
    "all_images = np.load(os.path.join(data_folder,'all_images_highfreq.npy'), allow_pickle=True)\n",
    "pv_data = np.load(pv_data_path, allow_pickle=True)\n",
    "\n",
    "# only pick out the relevant time period\n",
    "relevant_mask = (all_times>=start_date)&(all_times<end_date)\n",
    "all_times = all_times[relevant_mask]\n",
    "all_images = all_images[relevant_mask]\n",
    "pv_data = pv_data[start_date:end_date]\n",
    "\n",
    "n_images = all_times.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 6595,
     "status": "ok",
     "timestamp": 1599952937588,
     "user": {
      "displayName": "Andea Jewel Scott",
      "photoUrl": "",
      "userId": "15144577752603562183"
     },
     "user_tz": 420
    },
    "id": "DNaDTYlsaUPk",
    "outputId": "7ca39499-ecc0-4c68-ea22-62066a383436",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed 0/107714 images\n",
      "Processed 100/107714 images\n",
      "Processed 200/107714 images\n",
      "Processed 300/107714 images\n",
      "Processed 400/107714 images\n",
      "Processed 500/107714 images\n",
      "Processed 600/107714 images\n",
      "Processed 700/107714 images\n",
      "Processed 800/107714 images\n",
      "Processed 900/107714 images\n",
      "Processed 1000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:29.578239\n",
      "Processed 1100/107714 images\n",
      "Processed 1200/107714 images\n",
      "Processed 1300/107714 images\n",
      "Processed 1400/107714 images\n",
      "Processed 1500/107714 images\n",
      "Processed 1600/107714 images\n",
      "Processed 1700/107714 images\n",
      "Processed 1800/107714 images\n",
      "Processed 1900/107714 images\n",
      "Processed 2000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:27.019568\n",
      "Processed 2100/107714 images\n",
      "Processed 2200/107714 images\n",
      "Processed 2300/107714 images\n",
      "Processed 2400/107714 images\n",
      "Processed 2500/107714 images\n",
      "Processed 2600/107714 images\n",
      "Processed 2700/107714 images\n",
      "Processed 2800/107714 images\n",
      "Processed 2900/107714 images\n",
      "Processed 3000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:25.548638\n",
      "Processed 3100/107714 images\n",
      "Processed 3200/107714 images\n",
      "Processed 3300/107714 images\n",
      "Processed 3400/107714 images\n",
      "Processed 3500/107714 images\n",
      "Processed 3600/107714 images\n",
      "Processed 3700/107714 images\n",
      "Processed 3800/107714 images\n",
      "Processed 4000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:23.250191\n",
      "Processed 4100/107714 images\n",
      "Processed 4200/107714 images\n",
      "Processed 4300/107714 images\n",
      "Processed 4400/107714 images\n",
      "Processed 4500/107714 images\n",
      "Processed 4600/107714 images\n",
      "Processed 4800/107714 images\n",
      "Processed 4900/107714 images\n",
      "Processed 5000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:20.939100\n",
      "Processed 5100/107714 images\n",
      "Processed 5200/107714 images\n",
      "Processed 5300/107714 images\n",
      "Processed 5400/107714 images\n",
      "Processed 5500/107714 images\n",
      "Processed 5600/107714 images\n",
      "Processed 5700/107714 images\n",
      "Processed 5800/107714 images\n",
      "Processed 5900/107714 images\n",
      "Processed 6000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:19.685299\n",
      "Processed 6100/107714 images\n",
      "Processed 6200/107714 images\n",
      "Processed 6300/107714 images\n",
      "Processed 6400/107714 images\n",
      "Processed 6500/107714 images\n",
      "Processed 6600/107714 images\n",
      "Processed 6700/107714 images\n",
      "Processed 6800/107714 images\n",
      "Processed 6900/107714 images\n",
      "Processed 7000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:18.872781\n",
      "Processed 7100/107714 images\n",
      "Processed 7200/107714 images\n",
      "Processed 7300/107714 images\n",
      "Processed 7400/107714 images\n",
      "Processed 7500/107714 images\n",
      "Processed 7600/107714 images\n",
      "Processed 7700/107714 images\n",
      "Processed 7800/107714 images\n",
      "Processed 7900/107714 images\n",
      "Processed 8000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:18.033066\n",
      "Processed 8100/107714 images\n",
      "Processed 8200/107714 images\n",
      "Processed 8300/107714 images\n",
      "Processed 8400/107714 images\n",
      "Processed 8500/107714 images\n",
      "Processed 8600/107714 images\n",
      "Processed 8700/107714 images\n",
      "Processed 8800/107714 images\n",
      "Processed 8900/107714 images\n",
      "Processed 9000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:17.376954\n",
      "Processed 9100/107714 images\n",
      "Processed 9200/107714 images\n",
      "Processed 9300/107714 images\n",
      "Processed 9400/107714 images\n",
      "Processed 9500/107714 images\n",
      "Processed 9600/107714 images\n",
      "Processed 9700/107714 images\n",
      "Processed 9800/107714 images\n",
      "Processed 9900/107714 images\n",
      "Processed 10000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:17.051702\n",
      "Processed 10100/107714 images\n",
      "Processed 10200/107714 images\n",
      "Processed 10300/107714 images\n",
      "Processed 10400/107714 images\n",
      "Processed 10500/107714 images\n",
      "Processed 10600/107714 images\n",
      "Processed 10700/107714 images\n",
      "Processed 10800/107714 images\n",
      "Processed 10900/107714 images\n",
      "Processed 11000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:16.835969\n",
      "Processed 11100/107714 images\n",
      "Processed 11200/107714 images\n",
      "Processed 11300/107714 images\n",
      "Processed 11400/107714 images\n",
      "Processed 11500/107714 images\n",
      "Processed 11600/107714 images\n",
      "Processed 11700/107714 images\n",
      "Processed 11800/107714 images\n",
      "Processed 11900/107714 images\n",
      "Processed 12000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:16.431799\n",
      "Processed 12100/107714 images\n",
      "Processed 12200/107714 images\n",
      "Processed 12300/107714 images\n",
      "Processed 12400/107714 images\n",
      "Processed 12500/107714 images\n",
      "Processed 12600/107714 images\n",
      "Processed 12700/107714 images\n",
      "Processed 12800/107714 images\n",
      "Processed 12900/107714 images\n",
      "Processed 13000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:16.071016\n",
      "Processed 13100/107714 images\n",
      "Processed 13200/107714 images\n",
      "Processed 13300/107714 images\n",
      "Processed 13400/107714 images\n",
      "Processed 13500/107714 images\n",
      "Processed 13600/107714 images\n",
      "Processed 13700/107714 images\n",
      "Processed 13800/107714 images\n",
      "Processed 13900/107714 images\n",
      "Processed 14000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.951874\n",
      "Processed 14100/107714 images\n",
      "Processed 14200/107714 images\n",
      "Processed 14300/107714 images\n",
      "Processed 14400/107714 images\n",
      "Processed 14500/107714 images\n",
      "Processed 14600/107714 images\n",
      "Processed 14700/107714 images\n",
      "Processed 14800/107714 images\n",
      "Processed 14900/107714 images\n",
      "Processed 15000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.861101\n",
      "Processed 15100/107714 images\n",
      "Processed 15200/107714 images\n",
      "Processed 15300/107714 images\n",
      "Processed 15400/107714 images\n",
      "Processed 15500/107714 images\n",
      "Processed 15600/107714 images\n",
      "Processed 15700/107714 images\n",
      "Processed 15800/107714 images\n",
      "Processed 15900/107714 images\n",
      "Processed 16000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.751724\n",
      "Processed 16100/107714 images\n",
      "Processed 16200/107714 images\n",
      "Processed 16300/107714 images\n",
      "Processed 16400/107714 images\n",
      "Processed 16500/107714 images\n",
      "Processed 16600/107714 images\n",
      "Processed 16700/107714 images\n",
      "Processed 16800/107714 images\n",
      "Processed 16900/107714 images\n",
      "Processed 17000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.721828\n",
      "Processed 17100/107714 images\n",
      "Processed 17200/107714 images\n",
      "Processed 17300/107714 images\n",
      "Processed 17400/107714 images\n",
      "Processed 17500/107714 images\n",
      "Processed 17600/107714 images\n",
      "Processed 17700/107714 images\n",
      "Processed 17800/107714 images\n",
      "Processed 17900/107714 images\n",
      "Processed 18000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.590949\n",
      "Processed 18100/107714 images\n",
      "Processed 18200/107714 images\n",
      "Processed 18300/107714 images\n",
      "Processed 18400/107714 images\n",
      "Processed 18500/107714 images\n",
      "Processed 18600/107714 images\n",
      "Processed 18700/107714 images\n",
      "Processed 18800/107714 images\n",
      "Processed 18900/107714 images\n",
      "Processed 19000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.496342\n",
      "Processed 19100/107714 images\n",
      "Processed 19200/107714 images\n",
      "Processed 19300/107714 images\n",
      "Processed 19400/107714 images\n",
      "Processed 19500/107714 images\n",
      "Processed 19600/107714 images\n",
      "Processed 19700/107714 images\n",
      "Processed 19800/107714 images\n",
      "Processed 19900/107714 images\n",
      "Processed 20000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.396943\n",
      "Processed 20100/107714 images\n",
      "Processed 20200/107714 images\n",
      "Processed 20300/107714 images\n",
      "Processed 20400/107714 images\n",
      "Processed 20500/107714 images\n",
      "Processed 20600/107714 images\n",
      "Processed 20700/107714 images\n",
      "Processed 20800/107714 images\n",
      "Processed 20900/107714 images\n",
      "Processed 21000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.279037\n",
      "Processed 21100/107714 images\n",
      "Processed 21200/107714 images\n",
      "Processed 21300/107714 images\n",
      "Processed 21400/107714 images\n",
      "Processed 21500/107714 images\n",
      "Processed 21600/107714 images\n",
      "Processed 21700/107714 images\n",
      "Processed 21800/107714 images\n",
      "Processed 21900/107714 images\n",
      "Processed 22000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.160121\n",
      "Processed 22100/107714 images\n",
      "Processed 22200/107714 images\n",
      "Processed 22300/107714 images\n",
      "Processed 22400/107714 images\n",
      "Processed 22500/107714 images\n",
      "Processed 22600/107714 images\n",
      "Processed 22700/107714 images\n",
      "Processed 22800/107714 images\n",
      "Processed 22900/107714 images\n",
      "Processed 23000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.032991\n",
      "Processed 23100/107714 images\n",
      "Processed 23200/107714 images\n",
      "Processed 23300/107714 images\n",
      "Processed 23400/107714 images\n",
      "Processed 23500/107714 images\n",
      "Processed 23600/107714 images\n",
      "Processed 23700/107714 images\n",
      "Processed 23800/107714 images\n",
      "Processed 23900/107714 images\n",
      "Processed 24000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.063366\n",
      "Processed 24100/107714 images\n",
      "Processed 24200/107714 images\n",
      "Processed 24300/107714 images\n",
      "Processed 24400/107714 images\n",
      "Processed 24500/107714 images\n",
      "Processed 24600/107714 images\n",
      "Processed 24700/107714 images\n",
      "Processed 24800/107714 images\n",
      "Processed 24900/107714 images\n",
      "Processed 25000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.145705\n",
      "Processed 25100/107714 images\n",
      "Processed 25200/107714 images\n",
      "Processed 25300/107714 images\n",
      "Processed 25400/107714 images\n",
      "Processed 25500/107714 images\n",
      "Processed 25600/107714 images\n",
      "Processed 25700/107714 images\n",
      "Processed 25800/107714 images\n",
      "Processed 25900/107714 images\n",
      "Processed 26000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.096744\n",
      "Processed 26100/107714 images\n",
      "Processed 26200/107714 images\n",
      "Processed 26300/107714 images\n",
      "Processed 26400/107714 images\n",
      "Processed 26500/107714 images\n",
      "Processed 26600/107714 images\n",
      "Processed 26700/107714 images\n",
      "Processed 26800/107714 images\n",
      "Processed 26900/107714 images\n",
      "Processed 27000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.012076\n",
      "Processed 27100/107714 images\n",
      "Processed 27200/107714 images\n",
      "Processed 27300/107714 images\n",
      "Processed 27400/107714 images\n",
      "Processed 27500/107714 images\n",
      "Processed 27600/107714 images\n",
      "Processed 27700/107714 images\n",
      "Processed 27800/107714 images\n",
      "Processed 27900/107714 images\n",
      "Processed 28000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.976727\n",
      "Processed 28100/107714 images\n",
      "Processed 28200/107714 images\n",
      "Processed 28300/107714 images\n",
      "Processed 28400/107714 images\n",
      "Processed 28500/107714 images\n",
      "Processed 28600/107714 images\n",
      "Processed 28700/107714 images\n",
      "Processed 28800/107714 images\n",
      "Processed 28900/107714 images\n",
      "Processed 29000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.022778\n",
      "Processed 29100/107714 images\n",
      "Processed 29200/107714 images\n",
      "Processed 29300/107714 images\n",
      "Processed 29400/107714 images\n",
      "Processed 29500/107714 images\n",
      "Processed 29600/107714 images\n",
      "Processed 29700/107714 images\n",
      "Processed 29800/107714 images\n",
      "Processed 29900/107714 images\n",
      "Processed 30000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.990127\n",
      "Processed 30100/107714 images\n",
      "Processed 30200/107714 images\n",
      "Processed 30300/107714 images\n",
      "Processed 30400/107714 images\n",
      "Processed 30500/107714 images\n",
      "Processed 30600/107714 images\n",
      "Processed 30700/107714 images\n",
      "Processed 30800/107714 images\n",
      "Processed 30900/107714 images\n",
      "Processed 31000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.960927\n",
      "Processed 31100/107714 images\n",
      "Processed 31200/107714 images\n",
      "Processed 31300/107714 images\n",
      "Processed 31400/107714 images\n",
      "Processed 31500/107714 images\n",
      "Processed 31600/107714 images\n",
      "Processed 31700/107714 images\n",
      "Processed 31800/107714 images\n",
      "Processed 31900/107714 images\n",
      "Processed 32000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.937985\n",
      "Processed 32100/107714 images\n",
      "Processed 32200/107714 images\n",
      "Processed 32300/107714 images\n",
      "Processed 32400/107714 images\n",
      "Processed 32500/107714 images\n",
      "Processed 32600/107714 images\n",
      "Processed 32700/107714 images\n",
      "Processed 32800/107714 images\n",
      "Processed 32900/107714 images\n",
      "Processed 33000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.968513\n",
      "Processed 33100/107714 images\n",
      "Processed 33200/107714 images\n",
      "Processed 33300/107714 images\n",
      "Processed 33400/107714 images\n",
      "Processed 33500/107714 images\n",
      "Processed 33600/107714 images\n",
      "Processed 33700/107714 images\n",
      "Processed 33800/107714 images\n",
      "Processed 33900/107714 images\n",
      "Processed 34000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.025868\n",
      "Processed 34100/107714 images\n",
      "Processed 34200/107714 images\n",
      "Processed 34300/107714 images\n",
      "Processed 34400/107714 images\n",
      "Processed 34500/107714 images\n",
      "Processed 34600/107714 images\n",
      "Processed 34700/107714 images\n",
      "Processed 34800/107714 images\n",
      "Processed 34900/107714 images\n",
      "Processed 35000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.993744\n",
      "Processed 35100/107714 images\n",
      "Processed 35200/107714 images\n",
      "Processed 35300/107714 images\n",
      "Processed 35400/107714 images\n",
      "Processed 35500/107714 images\n",
      "Processed 35600/107714 images\n",
      "Processed 35700/107714 images\n",
      "Processed 35800/107714 images\n",
      "Processed 35900/107714 images\n",
      "Processed 36000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.967978\n",
      "Processed 36100/107714 images\n",
      "Processed 36200/107714 images\n",
      "Processed 36300/107714 images\n",
      "Processed 36400/107714 images\n",
      "Processed 36500/107714 images\n",
      "Processed 36600/107714 images\n",
      "Processed 36700/107714 images\n",
      "Processed 36800/107714 images\n",
      "Processed 36900/107714 images\n",
      "Processed 37000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.980320\n",
      "Processed 37100/107714 images\n",
      "Processed 37200/107714 images\n",
      "Processed 37300/107714 images\n",
      "Processed 37400/107714 images\n",
      "Processed 37500/107714 images\n",
      "Processed 37600/107714 images\n",
      "Processed 37700/107714 images\n",
      "Processed 37800/107714 images\n",
      "Processed 37900/107714 images\n",
      "Processed 38000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.999279\n",
      "Processed 38100/107714 images\n",
      "Processed 38200/107714 images\n",
      "Processed 38300/107714 images\n",
      "Processed 38400/107714 images\n",
      "Processed 38500/107714 images\n",
      "Processed 38600/107714 images\n",
      "Processed 38700/107714 images\n",
      "Processed 38800/107714 images\n",
      "Processed 38900/107714 images\n",
      "Processed 39000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.995653\n",
      "Processed 39100/107714 images\n",
      "Processed 39200/107714 images\n",
      "Processed 39300/107714 images\n",
      "Processed 39400/107714 images\n",
      "Processed 39500/107714 images\n",
      "Processed 39600/107714 images\n",
      "Processed 39700/107714 images\n",
      "Processed 39800/107714 images\n",
      "Processed 39900/107714 images\n",
      "Processed 40000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.123914\n",
      "Processed 40100/107714 images\n",
      "Processed 40200/107714 images\n",
      "Processed 40300/107714 images\n",
      "Processed 40400/107714 images\n",
      "Processed 40500/107714 images\n",
      "Processed 40600/107714 images\n",
      "Processed 40700/107714 images\n",
      "Processed 40800/107714 images\n",
      "Processed 40900/107714 images\n",
      "Processed 41000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.102259\n",
      "Processed 41100/107714 images\n",
      "Processed 41200/107714 images\n",
      "Processed 41300/107714 images\n",
      "Processed 41400/107714 images\n",
      "Processed 41500/107714 images\n",
      "Processed 41600/107714 images\n",
      "Processed 41700/107714 images\n",
      "Processed 41800/107714 images\n",
      "Processed 41900/107714 images\n",
      "Processed 42000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.064746\n",
      "Processed 42100/107714 images\n",
      "Processed 42200/107714 images\n",
      "Processed 42300/107714 images\n",
      "Processed 42400/107714 images\n",
      "Processed 42500/107714 images\n",
      "Processed 42600/107714 images\n",
      "Processed 42700/107714 images\n",
      "Processed 42800/107714 images\n",
      "Processed 42900/107714 images\n",
      "Processed 43000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:15.021609\n",
      "Processed 43100/107714 images\n",
      "Processed 43200/107714 images\n",
      "Processed 43300/107714 images\n",
      "Processed 43400/107714 images\n",
      "Processed 43500/107714 images\n",
      "Processed 43600/107714 images\n",
      "Processed 43700/107714 images\n",
      "Processed 43800/107714 images\n",
      "Processed 43900/107714 images\n",
      "Processed 44000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.955328\n",
      "Processed 44100/107714 images\n",
      "Processed 44200/107714 images\n",
      "Processed 44300/107714 images\n",
      "Processed 44400/107714 images\n",
      "Processed 44500/107714 images\n",
      "Processed 44600/107714 images\n",
      "Processed 44700/107714 images\n",
      "Processed 44800/107714 images\n",
      "Processed 44900/107714 images\n",
      "Processed 45000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.916221\n",
      "Processed 45100/107714 images\n",
      "Processed 45200/107714 images\n",
      "Processed 45300/107714 images\n",
      "Processed 45400/107714 images\n",
      "Processed 45500/107714 images\n",
      "Processed 45600/107714 images\n",
      "Processed 45700/107714 images\n",
      "Processed 45800/107714 images\n",
      "Processed 45900/107714 images\n",
      "Processed 46000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.862008\n",
      "Processed 46100/107714 images\n",
      "Processed 46200/107714 images\n",
      "Processed 46300/107714 images\n",
      "Processed 46400/107714 images\n",
      "Processed 46500/107714 images\n",
      "Processed 46600/107714 images\n",
      "Processed 46700/107714 images\n",
      "Processed 46800/107714 images\n",
      "Processed 46900/107714 images\n",
      "Processed 47000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.822870\n",
      "Processed 47100/107714 images\n",
      "Processed 47200/107714 images\n",
      "Processed 47300/107714 images\n",
      "Processed 47400/107714 images\n",
      "Processed 47500/107714 images\n",
      "Processed 47600/107714 images\n",
      "Processed 47700/107714 images\n",
      "Processed 47800/107714 images\n",
      "Processed 47900/107714 images\n",
      "Processed 48000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.808621\n",
      "Processed 48100/107714 images\n",
      "Processed 48200/107714 images\n",
      "Processed 48300/107714 images\n",
      "Processed 48400/107714 images\n",
      "Processed 48500/107714 images\n",
      "Processed 48600/107714 images\n",
      "Processed 48700/107714 images\n",
      "Processed 48800/107714 images\n",
      "Processed 48900/107714 images\n",
      "Processed 49000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.784190\n",
      "Processed 49100/107714 images\n",
      "Processed 49200/107714 images\n",
      "Processed 49300/107714 images\n",
      "Processed 49400/107714 images\n",
      "Processed 49500/107714 images\n",
      "Processed 49600/107714 images\n",
      "Processed 49700/107714 images\n",
      "Processed 49800/107714 images\n",
      "Processed 49900/107714 images\n",
      "Processed 50000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.759979\n",
      "Processed 50100/107714 images\n",
      "Processed 50200/107714 images\n",
      "Processed 50300/107714 images\n",
      "Processed 50400/107714 images\n",
      "Processed 50500/107714 images\n",
      "Processed 50600/107714 images\n",
      "Processed 50700/107714 images\n",
      "Processed 50800/107714 images\n",
      "Processed 50900/107714 images\n",
      "Processed 51000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.702782\n",
      "Processed 51100/107714 images\n",
      "Processed 51200/107714 images\n",
      "Processed 51300/107714 images\n",
      "Processed 51400/107714 images\n",
      "Processed 51500/107714 images\n",
      "Processed 51600/107714 images\n",
      "Processed 51700/107714 images\n",
      "Processed 51800/107714 images\n",
      "Processed 51900/107714 images\n",
      "Processed 52000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.683681\n",
      "Processed 52100/107714 images\n",
      "Processed 52200/107714 images\n",
      "Processed 52300/107714 images\n",
      "Processed 52400/107714 images\n",
      "Processed 52500/107714 images\n",
      "Processed 52600/107714 images\n",
      "Processed 52700/107714 images\n",
      "Processed 52800/107714 images\n",
      "Processed 52900/107714 images\n",
      "Processed 53000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.632802\n",
      "Processed 53100/107714 images\n",
      "Processed 53200/107714 images\n",
      "Processed 53300/107714 images\n",
      "Processed 53400/107714 images\n",
      "Processed 53500/107714 images\n",
      "Processed 53600/107714 images\n",
      "Processed 53700/107714 images\n",
      "Processed 53800/107714 images\n",
      "Processed 53900/107714 images\n",
      "Processed 54000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.587466\n",
      "Processed 54100/107714 images\n",
      "Processed 54200/107714 images\n",
      "Processed 54300/107714 images\n",
      "Processed 54400/107714 images\n",
      "Processed 54500/107714 images\n",
      "Processed 54600/107714 images\n",
      "Processed 54700/107714 images\n",
      "Processed 54800/107714 images\n",
      "Processed 54900/107714 images\n",
      "Processed 55000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.547448\n",
      "Processed 55100/107714 images\n",
      "Processed 55200/107714 images\n",
      "Processed 55300/107714 images\n",
      "Processed 55400/107714 images\n",
      "Processed 55500/107714 images\n",
      "Processed 55600/107714 images\n",
      "Processed 55700/107714 images\n",
      "Processed 55800/107714 images\n",
      "Processed 55900/107714 images\n",
      "Processed 56000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.549398\n",
      "Processed 56100/107714 images\n",
      "Processed 56200/107714 images\n",
      "Processed 56300/107714 images\n",
      "Processed 56400/107714 images\n",
      "Processed 56500/107714 images\n",
      "Processed 56600/107714 images\n",
      "Processed 56700/107714 images\n",
      "Processed 56800/107714 images\n",
      "Processed 56900/107714 images\n",
      "Processed 57000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.513386\n",
      "Processed 57100/107714 images\n",
      "Processed 57200/107714 images\n",
      "Processed 57300/107714 images\n",
      "Processed 57400/107714 images\n",
      "Processed 57500/107714 images\n",
      "Processed 57600/107714 images\n",
      "Processed 57700/107714 images\n",
      "Processed 57800/107714 images\n",
      "Processed 57900/107714 images\n",
      "Processed 58000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.482275\n",
      "Processed 58100/107714 images\n",
      "Processed 58200/107714 images\n",
      "Processed 58300/107714 images\n",
      "Processed 58400/107714 images\n",
      "Processed 58500/107714 images\n",
      "Processed 58600/107714 images\n",
      "Processed 58700/107714 images\n",
      "Processed 58800/107714 images\n",
      "Processed 58900/107714 images\n",
      "Processed 59000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.425650\n",
      "Processed 59100/107714 images\n",
      "Processed 59200/107714 images\n",
      "Processed 59300/107714 images\n",
      "Processed 59400/107714 images\n",
      "Processed 59500/107714 images\n",
      "Processed 59600/107714 images\n",
      "Processed 59700/107714 images\n",
      "Processed 59800/107714 images\n",
      "Processed 59900/107714 images\n",
      "Processed 60000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.364444\n",
      "Processed 60100/107714 images\n",
      "Processed 60200/107714 images\n",
      "Processed 60300/107714 images\n",
      "Processed 60400/107714 images\n",
      "Processed 60500/107714 images\n",
      "Processed 60600/107714 images\n",
      "Processed 60700/107714 images\n",
      "Processed 60800/107714 images\n",
      "Processed 60900/107714 images\n",
      "Processed 61000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.301114\n",
      "Processed 61100/107714 images\n",
      "Processed 61200/107714 images\n",
      "Processed 61300/107714 images\n",
      "Processed 61400/107714 images\n",
      "Processed 61500/107714 images\n",
      "Processed 61600/107714 images\n",
      "Processed 61700/107714 images\n",
      "Processed 61800/107714 images\n",
      "Processed 61900/107714 images\n",
      "Processed 62000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.224717\n",
      "Processed 62100/107714 images\n",
      "Processed 62200/107714 images\n",
      "Processed 62300/107714 images\n",
      "Processed 62400/107714 images\n",
      "Processed 62500/107714 images\n",
      "Processed 62600/107714 images\n",
      "Processed 62700/107714 images\n",
      "Processed 62800/107714 images\n",
      "Processed 62900/107714 images\n",
      "Processed 63000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.144006\n",
      "Processed 63100/107714 images\n",
      "Processed 63200/107714 images\n",
      "Processed 63300/107714 images\n",
      "Processed 63400/107714 images\n",
      "Processed 63500/107714 images\n",
      "Processed 63600/107714 images\n",
      "Processed 63700/107714 images\n",
      "Processed 63800/107714 images\n",
      "Processed 63900/107714 images\n",
      "Processed 64000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.080927\n",
      "Processed 64100/107714 images\n",
      "Processed 64200/107714 images\n",
      "Processed 64300/107714 images\n",
      "Processed 64400/107714 images\n",
      "Processed 64500/107714 images\n",
      "Processed 64600/107714 images\n",
      "Processed 64700/107714 images\n",
      "Processed 64800/107714 images\n",
      "Processed 64900/107714 images\n",
      "Processed 65000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:14.008148\n",
      "Processed 65100/107714 images\n",
      "Processed 65200/107714 images\n",
      "Processed 65300/107714 images\n",
      "Processed 65400/107714 images\n",
      "Processed 65500/107714 images\n",
      "Processed 65600/107714 images\n",
      "Processed 65700/107714 images\n",
      "Processed 65800/107714 images\n",
      "Processed 65900/107714 images\n",
      "Processed 66000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.949123\n",
      "Processed 66100/107714 images\n",
      "Processed 66200/107714 images\n",
      "Processed 66300/107714 images\n",
      "Processed 66400/107714 images\n",
      "Processed 66500/107714 images\n",
      "Processed 66600/107714 images\n",
      "Processed 66700/107714 images\n",
      "Processed 66800/107714 images\n",
      "Processed 66900/107714 images\n",
      "Processed 67000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.870183\n",
      "Processed 67100/107714 images\n",
      "Processed 67200/107714 images\n",
      "Processed 67300/107714 images\n",
      "Processed 67400/107714 images\n",
      "Processed 67500/107714 images\n",
      "Processed 67600/107714 images\n",
      "Processed 67700/107714 images\n",
      "Processed 67800/107714 images\n",
      "Processed 67900/107714 images\n",
      "Processed 68000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.799373\n",
      "Processed 68100/107714 images\n",
      "Processed 68200/107714 images\n",
      "Processed 68300/107714 images\n",
      "Processed 68400/107714 images\n",
      "Processed 68500/107714 images\n",
      "Processed 68600/107714 images\n",
      "Processed 68700/107714 images\n",
      "Processed 68800/107714 images\n",
      "Processed 68900/107714 images\n",
      "Processed 69000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.724180\n",
      "Processed 69100/107714 images\n",
      "Processed 69200/107714 images\n",
      "Processed 69300/107714 images\n",
      "Processed 69400/107714 images\n",
      "Processed 69500/107714 images\n",
      "Processed 69600/107714 images\n",
      "Processed 69700/107714 images\n",
      "Processed 69800/107714 images\n",
      "Processed 69900/107714 images\n",
      "Processed 70000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.665055\n",
      "Processed 70100/107714 images\n",
      "Processed 70200/107714 images\n",
      "Processed 70300/107714 images\n",
      "Processed 70400/107714 images\n",
      "Processed 70500/107714 images\n",
      "Processed 70600/107714 images\n",
      "Processed 70700/107714 images\n",
      "Processed 70800/107714 images\n",
      "Processed 70900/107714 images\n",
      "Processed 71000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.598380\n",
      "Processed 71100/107714 images\n",
      "Processed 71200/107714 images\n",
      "Processed 71300/107714 images\n",
      "Processed 71400/107714 images\n",
      "Processed 71500/107714 images\n",
      "Processed 71600/107714 images\n",
      "Processed 71700/107714 images\n",
      "Processed 71800/107714 images\n",
      "Processed 71900/107714 images\n",
      "Processed 72000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.553676\n",
      "Processed 72100/107714 images\n",
      "Processed 72200/107714 images\n",
      "Processed 72300/107714 images\n",
      "Processed 72400/107714 images\n",
      "Processed 72500/107714 images\n",
      "Processed 72600/107714 images\n",
      "Processed 72700/107714 images\n",
      "Processed 72800/107714 images\n",
      "Processed 72900/107714 images\n",
      "Processed 73000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.502069\n",
      "Processed 73100/107714 images\n",
      "Processed 73200/107714 images\n",
      "Processed 73400/107714 images\n",
      "Processed 73500/107714 images\n",
      "Processed 73600/107714 images\n",
      "Processed 73700/107714 images\n",
      "Processed 73800/107714 images\n",
      "Processed 73900/107714 images\n",
      "Processed 74000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.446333\n",
      "Processed 74100/107714 images\n",
      "Processed 74200/107714 images\n",
      "Processed 74300/107714 images\n",
      "Processed 74400/107714 images\n",
      "Processed 74500/107714 images\n",
      "Processed 74600/107714 images\n",
      "Processed 74700/107714 images\n",
      "Processed 74800/107714 images\n",
      "Processed 74900/107714 images\n",
      "Processed 75000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.392806\n",
      "Processed 75100/107714 images\n",
      "Processed 75200/107714 images\n",
      "Processed 75300/107714 images\n",
      "Processed 75400/107714 images\n",
      "Processed 75500/107714 images\n",
      "Processed 75600/107714 images\n",
      "Processed 75700/107714 images\n",
      "Processed 75800/107714 images\n",
      "Processed 75900/107714 images\n",
      "Processed 76000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.345682\n",
      "Processed 76100/107714 images\n",
      "Processed 76200/107714 images\n",
      "Processed 76300/107714 images\n",
      "Processed 76400/107714 images\n",
      "Processed 76500/107714 images\n",
      "Processed 76600/107714 images\n",
      "Processed 76700/107714 images\n",
      "Processed 76800/107714 images\n",
      "Processed 76900/107714 images\n",
      "Processed 77000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.289420\n",
      "Processed 77100/107714 images\n",
      "Processed 77200/107714 images\n",
      "Processed 77300/107714 images\n",
      "Processed 77400/107714 images\n",
      "Processed 77500/107714 images\n",
      "Processed 77600/107714 images\n",
      "Processed 77700/107714 images\n",
      "Processed 77800/107714 images\n",
      "Processed 77900/107714 images\n",
      "Processed 78000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.246612\n",
      "Processed 78100/107714 images\n",
      "Processed 78200/107714 images\n",
      "Processed 78300/107714 images\n",
      "Processed 78400/107714 images\n",
      "Processed 78500/107714 images\n",
      "Processed 78600/107714 images\n",
      "Processed 78700/107714 images\n",
      "Processed 78800/107714 images\n",
      "Processed 78900/107714 images\n",
      "Processed 79000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.190968\n",
      "Processed 79100/107714 images\n",
      "Processed 79200/107714 images\n",
      "Processed 79300/107714 images\n",
      "Processed 79400/107714 images\n",
      "Processed 79500/107714 images\n",
      "Processed 79600/107714 images\n",
      "Processed 79700/107714 images\n",
      "Processed 79800/107714 images\n",
      "Processed 79900/107714 images\n",
      "Processed 80000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.139355\n",
      "Processed 80100/107714 images\n",
      "Processed 80200/107714 images\n",
      "Processed 80300/107714 images\n",
      "Processed 80400/107714 images\n",
      "Processed 80500/107714 images\n",
      "Processed 80600/107714 images\n",
      "Processed 80700/107714 images\n",
      "Processed 80800/107714 images\n",
      "Processed 80900/107714 images\n",
      "Processed 81000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.104423\n",
      "Processed 81100/107714 images\n",
      "Processed 81200/107714 images\n",
      "Processed 81300/107714 images\n",
      "Processed 81400/107714 images\n",
      "Processed 81500/107714 images\n",
      "Processed 81600/107714 images\n",
      "Processed 81700/107714 images\n",
      "Processed 81800/107714 images\n",
      "Processed 81900/107714 images\n",
      "Processed 82000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.053677\n",
      "Processed 82100/107714 images\n",
      "Processed 82200/107714 images\n",
      "Processed 82300/107714 images\n",
      "Processed 82400/107714 images\n",
      "Processed 82500/107714 images\n",
      "Processed 82600/107714 images\n",
      "Processed 82700/107714 images\n",
      "Processed 82800/107714 images\n",
      "Processed 82900/107714 images\n",
      "Processed 83000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:13.003871\n",
      "Processed 83100/107714 images\n",
      "Processed 83200/107714 images\n",
      "Processed 83300/107714 images\n",
      "Processed 83400/107714 images\n",
      "Processed 83500/107714 images\n",
      "Processed 83600/107714 images\n",
      "Processed 83700/107714 images\n",
      "Processed 83800/107714 images\n",
      "Processed 83900/107714 images\n",
      "Processed 84000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.962396\n",
      "Processed 84100/107714 images\n",
      "Processed 84200/107714 images\n",
      "Processed 84300/107714 images\n",
      "Processed 84400/107714 images\n",
      "Processed 84500/107714 images\n",
      "Processed 84600/107714 images\n",
      "Processed 84700/107714 images\n",
      "Processed 84800/107714 images\n",
      "Processed 84900/107714 images\n",
      "Processed 85000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.918763\n",
      "Processed 85100/107714 images\n",
      "Processed 85200/107714 images\n",
      "Processed 85300/107714 images\n",
      "Processed 85400/107714 images\n",
      "Processed 85500/107714 images\n",
      "Processed 85600/107714 images\n",
      "Processed 85700/107714 images\n",
      "Processed 85800/107714 images\n",
      "Processed 85900/107714 images\n",
      "Processed 86000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.883519\n",
      "Processed 86100/107714 images\n",
      "Processed 86200/107714 images\n",
      "Processed 86300/107714 images\n",
      "Processed 86400/107714 images\n",
      "Processed 86500/107714 images\n",
      "Processed 86600/107714 images\n",
      "Processed 86700/107714 images\n",
      "Processed 86800/107714 images\n",
      "Processed 86900/107714 images\n",
      "Processed 87000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.838122\n",
      "Processed 87100/107714 images\n",
      "Processed 87200/107714 images\n",
      "Processed 87300/107714 images\n",
      "Processed 87400/107714 images\n",
      "Processed 87500/107714 images\n",
      "Processed 87600/107714 images\n",
      "Processed 87700/107714 images\n",
      "Processed 87800/107714 images\n",
      "Processed 87900/107714 images\n",
      "Processed 88000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.796505\n",
      "Processed 88100/107714 images\n",
      "Processed 88200/107714 images\n",
      "Processed 88300/107714 images\n",
      "Processed 88400/107714 images\n",
      "Processed 88500/107714 images\n",
      "Processed 88600/107714 images\n",
      "Processed 88800/107714 images\n",
      "Processed 88900/107714 images\n",
      "Processed 89000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.757408\n",
      "Processed 89100/107714 images\n",
      "Processed 89200/107714 images\n",
      "Processed 89300/107714 images\n",
      "Processed 89400/107714 images\n",
      "Processed 89500/107714 images\n",
      "Processed 89600/107714 images\n",
      "Processed 89700/107714 images\n",
      "Processed 89800/107714 images\n",
      "Processed 89900/107714 images\n",
      "Processed 90000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.719644\n",
      "Processed 90100/107714 images\n",
      "Processed 90200/107714 images\n",
      "Processed 90300/107714 images\n",
      "Processed 90400/107714 images\n",
      "Processed 90500/107714 images\n",
      "Processed 90600/107714 images\n",
      "Processed 90700/107714 images\n",
      "Processed 90800/107714 images\n",
      "Processed 90900/107714 images\n",
      "Processed 91000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.684597\n",
      "Processed 91100/107714 images\n",
      "Processed 91200/107714 images\n",
      "Processed 91300/107714 images\n",
      "Processed 91400/107714 images\n",
      "Processed 91500/107714 images\n",
      "Processed 91600/107714 images\n",
      "Processed 91700/107714 images\n",
      "Processed 91800/107714 images\n",
      "Processed 91900/107714 images\n",
      "Processed 92000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.648196\n",
      "Processed 92100/107714 images\n",
      "Processed 92200/107714 images\n",
      "Processed 92300/107714 images\n",
      "Processed 92400/107714 images\n",
      "Processed 92500/107714 images\n",
      "Processed 92600/107714 images\n",
      "Processed 92700/107714 images\n",
      "Processed 92800/107714 images\n",
      "Processed 92900/107714 images\n",
      "Processed 93000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.623006\n",
      "Processed 93100/107714 images\n",
      "Processed 93200/107714 images\n",
      "Processed 93300/107714 images\n",
      "Processed 93400/107714 images\n",
      "Processed 93500/107714 images\n",
      "Processed 93600/107714 images\n",
      "Processed 93700/107714 images\n",
      "Processed 93800/107714 images\n",
      "Processed 93900/107714 images\n",
      "Processed 94000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.591080\n",
      "Processed 94100/107714 images\n",
      "Processed 94200/107714 images\n",
      "Processed 94300/107714 images\n",
      "Processed 94400/107714 images\n",
      "Processed 94500/107714 images\n",
      "Processed 94600/107714 images\n",
      "Processed 94700/107714 images\n",
      "Processed 94800/107714 images\n",
      "Processed 94900/107714 images\n",
      "Processed 95000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.558859\n",
      "Processed 95100/107714 images\n",
      "Processed 95200/107714 images\n",
      "Processed 95300/107714 images\n",
      "Processed 95400/107714 images\n",
      "Processed 95500/107714 images\n",
      "Processed 95600/107714 images\n",
      "Processed 95700/107714 images\n",
      "Processed 95800/107714 images\n",
      "Processed 95900/107714 images\n",
      "Processed 96000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.520413\n",
      "Processed 96100/107714 images\n",
      "Processed 96200/107714 images\n",
      "Processed 96300/107714 images\n",
      "Processed 96400/107714 images\n",
      "Processed 96500/107714 images\n",
      "Processed 96600/107714 images\n",
      "Processed 96700/107714 images\n",
      "Processed 96800/107714 images\n",
      "Processed 96900/107714 images\n",
      "Processed 97000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.493191\n",
      "Processed 97100/107714 images\n",
      "Processed 97200/107714 images\n",
      "Processed 97300/107714 images\n",
      "Processed 97400/107714 images\n",
      "Processed 97500/107714 images\n",
      "Processed 97600/107714 images\n",
      "Processed 97700/107714 images\n",
      "Processed 97800/107714 images\n",
      "Processed 97900/107714 images\n",
      "Processed 98000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.458624\n",
      "Processed 98100/107714 images\n",
      "Processed 98200/107714 images\n",
      "Processed 98300/107714 images\n",
      "Processed 98400/107714 images\n",
      "Processed 98500/107714 images\n",
      "Processed 98600/107714 images\n",
      "Processed 98700/107714 images\n",
      "Processed 98800/107714 images\n",
      "Processed 98900/107714 images\n",
      "Processed 99000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.420419\n",
      "Processed 99100/107714 images\n",
      "Processed 99200/107714 images\n",
      "Processed 99300/107714 images\n",
      "Processed 99400/107714 images\n",
      "Processed 99500/107714 images\n",
      "Processed 99600/107714 images\n",
      "Processed 99700/107714 images\n",
      "Processed 99800/107714 images\n",
      "Processed 99900/107714 images\n",
      "Processed 100000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.390695\n",
      "Processed 100100/107714 images\n",
      "Processed 100200/107714 images\n",
      "Processed 100300/107714 images\n",
      "Processed 100400/107714 images\n",
      "Processed 100500/107714 images\n",
      "Processed 100600/107714 images\n",
      "Processed 100700/107714 images\n",
      "Processed 100800/107714 images\n",
      "Processed 100900/107714 images\n",
      "Processed 101000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.356871\n",
      "Processed 101100/107714 images\n",
      "Processed 101200/107714 images\n",
      "Processed 101300/107714 images\n",
      "Processed 101400/107714 images\n",
      "Processed 101500/107714 images\n",
      "Processed 101600/107714 images\n",
      "Processed 101700/107714 images\n",
      "Processed 101800/107714 images\n",
      "Processed 101900/107714 images\n",
      "Processed 102000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.329633\n",
      "Processed 102100/107714 images\n",
      "Processed 102200/107714 images\n",
      "Processed 102300/107714 images\n",
      "Processed 102400/107714 images\n",
      "Processed 102500/107714 images\n",
      "Processed 102600/107714 images\n",
      "Processed 102700/107714 images\n",
      "Processed 102800/107714 images\n",
      "Processed 102900/107714 images\n",
      "Processed 103000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.305127\n",
      "Processed 103100/107714 images\n",
      "Processed 103200/107714 images\n",
      "Processed 103300/107714 images\n",
      "Processed 103400/107714 images\n",
      "Processed 103500/107714 images\n",
      "Processed 103600/107714 images\n",
      "Processed 103700/107714 images\n",
      "Processed 103800/107714 images\n",
      "Processed 103900/107714 images\n",
      "Processed 104000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.275486\n",
      "Processed 104100/107714 images\n",
      "Processed 104200/107714 images\n",
      "Processed 104300/107714 images\n",
      "Processed 104400/107714 images\n",
      "Processed 104500/107714 images\n",
      "Processed 104600/107714 images\n",
      "Processed 104700/107714 images\n",
      "Processed 104800/107714 images\n",
      "Processed 104900/107714 images\n",
      "Processed 105000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.247086\n",
      "Processed 105100/107714 images\n",
      "Processed 105200/107714 images\n",
      "Processed 105300/107714 images\n",
      "Processed 105400/107714 images\n",
      "Processed 105500/107714 images\n",
      "Processed 105600/107714 images\n",
      "Processed 105700/107714 images\n",
      "Processed 105800/107714 images\n",
      "Processed 105900/107714 images\n",
      "Processed 106000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.214406\n",
      "Processed 106100/107714 images\n",
      "Processed 106200/107714 images\n",
      "Processed 106300/107714 images\n",
      "Processed 106400/107714 images\n",
      "Processed 106500/107714 images\n",
      "Processed 106600/107714 images\n",
      "Processed 106700/107714 images\n",
      "Processed 106800/107714 images\n",
      "Processed 106900/107714 images\n",
      "Processed 107000/107714 images\n",
      "For sampling frequency: 2 minutes\n",
      "Expected finishing time: 2025-02-22 15:54:12.187400\n",
      "Processed 107100/107714 images\n",
      "Processed 107200/107714 images\n",
      "Processed 107300/107714 images\n",
      "Processed 107400/107714 images\n",
      "Processed 107500/107714 images\n",
      "Processed 107600/107714 images\n",
      "Processed 107700/107714 images\n"
     ]
    }
   ],
   "source": [
    "for sampling_interval in sampling_interval_all:\n",
    "    # Initialize variables to save pv values\n",
    "    image_log = np.zeros([n_images, stack_height + 1] + output_img_shape, dtype='uint8')  # (N, 16, 64, 64, 3)\n",
    "    image_pred = np.zeros([n_images, forecast_horizon] + output_img_shape, dtype='uint8')     # (N, 5, 64, 64, 3)\n",
    "    pv_log = np.zeros((n_images, stack_height + 1))                                       # (N, 16)\n",
    "    pv_pred = np.zeros((n_images, forecast_horizon))                                     # (N, 5) 修改为二维数组\n",
    "    validity_mask = np.ones(n_images, dtype=bool)\n",
    "    tic = time.process_time()\n",
    "    last_valid_index = 0\n",
    "\n",
    "    # Define the sampling interval in timedelta format\n",
    "    sampling_interval_td = datetime.timedelta(minutes=sampling_interval) - datetime.timedelta(seconds=1)\n",
    "\n",
    "    for i in range(0, n_images):\n",
    "        # Check if the specified sampling frequency is met\n",
    "        if all_times[i] - all_times[last_valid_index] > sampling_interval_td:\n",
    "            # Collecting ground truth for predicted value\n",
    "            for j in range(forecast_horizon):  # 收集未来 forecast_horizon 步的 PV 预测值\n",
    "                pred_time = all_times[i] + datetime.timedelta(minutes=j + 1)  # t+1 到 t+forecast_horizon\n",
    "                pv_pred_idx = find_time_within_nparray(pv_data.index, pred_time)\n",
    "                if pv_pred_idx is None:  # 如果预测的某个时间点找不到对应的 PV 值\n",
    "                    validity_mask[i] = False\n",
    "                    # print(f\"{all_times[i]} has no PV pred at t+{j+1}\")\n",
    "                    break\n",
    "                else:\n",
    "                    pv_pred[i, j] = pv_data.iloc[pv_pred_idx]  # 将预测值存入 pv_pred[i, j]\n",
    "\n",
    "            if not validity_mask[i]:  # 如果当前样本无效，跳过后续处理\n",
    "                continue\n",
    "\n",
    "            # Collecting image log and PV log\n",
    "            for j in range(stack_height + 1):  # 历史图像和 PV 值从 t-15 到 t\n",
    "                log_time = all_times[i] - datetime.timedelta(minutes=j)\n",
    "                log_time_idx = find_time_within_nparray(all_times, log_time)\n",
    "                if log_time_idx is None:\n",
    "                    validity_mask[i] = False\n",
    "                    # print(f\"{all_times[i]} has no image log\")\n",
    "                    break\n",
    "                else:\n",
    "                    image_log[i, j] = all_images[log_time_idx]\n",
    "\n",
    "                pv_log_idx = find_time_within_nparray(pv_data.index, log_time)\n",
    "                if pv_log_idx is None:\n",
    "                    validity_mask[i] = False\n",
    "                    # print(f\"{all_times[i]} has no PV log\")\n",
    "                    break\n",
    "                else:\n",
    "                    pv_log[i, j] = pv_data.iloc[pv_log_idx]\n",
    "\n",
    "            # Collecting future images for prediction\n",
    "            for j in range(forecast_horizon):  # 未来图像从 t+1 到 t+forecast_horizon\n",
    "                pred_image_time = all_times[i] + datetime.timedelta(minutes=j + 1)\n",
    "                pred_image_time_idx = find_time_within_nparray(all_times, pred_image_time)\n",
    "                if pred_image_time_idx is None:\n",
    "                    validity_mask[i] = False\n",
    "                    # print(f\"{all_times[i]} has no image pred\")\n",
    "                    break\n",
    "                else:\n",
    "                    image_pred[i, j] = all_images[pred_image_time_idx]\n",
    "\n",
    "        else:  # 如果不符合采样频率要求，丢弃该样本\n",
    "            validity_mask[i] = False\n",
    "\n",
    "        if validity_mask[i]:\n",
    "            last_valid_index = i\n",
    "\n",
    "        # 提示当前进度\n",
    "        if i % 100 == 0:\n",
    "            print(f\"Processed {i}/{len(all_times)} images\")\n",
    "            if i % 1000 == 0 and i > 0:\n",
    "                print(f\"For sampling frequency: {sampling_interval} minutes\")\n",
    "                print(\"Expected finishing time:\", \n",
    "                      datetime.datetime.now() + datetime.timedelta(seconds=(time.process_time() - tic) * (len(all_times) / i - 1)))\n",
    "\n",
    "    # 只保留有效的样本\n",
    "    all_times = all_times[validity_mask]\n",
    "    image_log = image_log[validity_mask]\n",
    "    image_pred = image_pred[validity_mask]\n",
    "    pv_log = pv_log[validity_mask]\n",
    "    pv_pred = pv_pred[validity_mask]  # 现在 pv_pred 是二维数组\n",
    "    \n",
    "    # Store information\n",
    "    \n",
    "    store_trainval_test(all_times,image_log,image_pred,pv_log,pv_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_h5py(base_path):\n",
    "    # 更新后的 np_files 字典，包含 images_pred\n",
    "    np_files = {\n",
    "        'trainval': [\n",
    "            base_path + 'pv_log_trainval.npy', \n",
    "            base_path + 'pv_pred_trainval.npy', \n",
    "            base_path + 'image_log_trainval.npy', \n",
    "            base_path + 'image_pred_trainval.npy'  # 新增 images_pred\n",
    "        ],\n",
    "        'test': [\n",
    "            base_path + 'image_log_test.npy', \n",
    "            base_path + 'image_pred_test.npy',     # 新增 images_pred\n",
    "            base_path + 'pv_log_test.npy', \n",
    "            base_path + 'pv_pred_test.npy'\n",
    "        ]\n",
    "    }\n",
    "\n",
    "    # 创建 HDF5 文件并写入数据\n",
    "    with h5py.File(f'./data/video_data/video_5min.hdf5', 'w') as f:\n",
    "        for group_name, files in np_files.items():\n",
    "            group = f.create_group(group_name)  # 创建组（trainval 或 test）\n",
    "            for file in files:\n",
    "                if os.path.exists(file):  # 检查文件是否存在\n",
    "                    data = np.load(file)  # 加载 NumPy 数据\n",
    "                    dataset_name = os.path.splitext(os.path.basename(file))[0]  # 提取文件名作为数据集名称\n",
    "                    group.create_dataset(dataset_name, data=data)  # 在组中创建数据集\n",
    "                else:\n",
    "                    print(f\"Warning: File {file} does not exist. Skipping...\")\n",
    "\n",
    "create_h5py(\"./data/npy_before_hdf5/All/\")"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "data_preprocess_forecast.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "solar",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
